# coding=utf-8

import threading
from datetime import datetime

from difflib import SequenceMatcher


def p(str1):
    if 0:
        print("%s" % str1)


def wordcount(dict_, str1):

    found = False
    for key in dict_.keys():
        ret = SequenceMatcher(None, key, str1).quick_ratio()
        if ret >= 0.8 :
            found = True
            dict_[key] += 1
            return
    if not found:
        dict_[str1] = 1


def do_str_match(lines, start, end, dict_):

    global statistics
    
    p("info: do_str_match(%s, %s) BEGIN" % (start, end))
    
    for i in range(start, end + 1):
#         p("info: do_str_match: %s" % i)
        wordcount(dict_, lines[i])

    p("info: do_str_match(%s, %s) DONE" % (start, end))


def do_combine_dict(dict1, dict2):
#     if len(dict1) < len(dict2):
#         return do_combine_dict(dict2, dict1)
    
    for key1 in list(dict1.keys()):
        for key2 in dict2.keys():
            ret = SequenceMatcher(None, key1, key2).quick_ratio()
            if ret > 0.8 :
                dict1[key1] += dict2[key2]
                del dict2[key2]
                break
            
    for key, value in dict2.items():
        dict1[key] = value
    
    
if __name__ == '__main__':

    # 1.读文件
    with open('1.copy.txt', 'rb') as f:
        lines = f.readlines()
    
    # 2.分段
    nrows = len(lines)
    count = 30
    part = nrows // count
    
    # 3.分段统计
    threads = []
    dict_list = []
    for i in range(0, count):
        begin = i * part
        if i != count - 1:
            end = (i + 1) * part - 1
        else:
            end = nrows - 1
        
        d_new = dict()
        dict_list.append(d_new)
        t = threading.Thread(target=do_str_match, args=(lines, begin, end, d_new))
        threads.append(t)

    print(datetime.now())
    
    for t in threads:
        t.start()
          
    for t in threads:
        t.join()

    print(datetime.now())
    
    # 校验总和
#     i = 0
#     for d_ in dict_list:
#         sum1 = 0;
#         for v in d_.values():
#             sum1 += v
#         print("(%s, %s, %s)" % (i, len(d_), sum1))
#         i += 1
#     print("##################################")
    
    # 4.合并
    while True:
        count = len(dict_list)
        if count == 1:
            break
        
        threads = []
        dict_list_new = []
        for i in range(0, count // 2):
            p("info: combine(%s, %s)" % (i * 2, i * 2 + 1))
            if len(dict_list[i * 2]) >= len(dict_list[i * 2 + 1]):
                t = threading.Thread(target=do_combine_dict, args=(dict_list[i * 2], dict_list[i * 2 + 1]))
                dict_list_new.append(dict_list[i * 2])
            else:
                t = threading.Thread(target=do_combine_dict, args=(dict_list[i * 2 + 1], dict_list[i * 2]))
                dict_list_new.append(dict_list[i * 2 + 1])
            threads.append(t)
            
        for t in threads:
            t.start()
              
        for t in threads:
            t.join()
        
        if count % 2 == 1:
            dict_list_new.append(dict_list[count - 1])
            
        dict_list = dict_list_new
        
        # 校验总和
#         i = 0
#         for d_ in dict_list:
#             sum1 = 0;
#             for v in d_.values():
#                 sum1 += v
#             print("(%s, %s, %s)" % (i, len(d_), sum1))
#             i += 1
#         print("##################################")

    print(datetime.now())
    
    # 5.排序
    result = sorted(dict_list[0].items(), key=lambda x:x[1], reverse=True)
    
    print(datetime.now())
    
#     print(result)
    
    # 6.校验总和
    sum1 = 0;
    for v in result:
        sum1 += v[1]
    print("info: sum=%s" % sum1)
        
    print("*** OVER ***")

